import copy
import json
import time

import scrapy
from functools import reduce
from shiGuang.items import ShiguangCareItem
from shiGuang.items import ShiguangTypeItem

class ShiSpider(scrapy.Spider):
    name = 'shi'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['www.xxx.com'] # 更换列表地址
    companyId = '2d4973ba6adf4a1e9e6f12c773a47e1c' # 公司id(替换真实值)(不用管)
    userId = 'cfa4260f6bbf4d4a99b94f2ff1c507db' # 员工id(替换真实值)(不用管)


    def parse(self, response):
        """
        :title : 取数据
        :param response:
        :return:
        """
        if response.status == 200: # 判断页面是否进入成功
            # 处理类型数据
            json_list_care = response.xpath('//*[@id="__NEXT_DATA__"]/text()').get()
            list_care = json.loads(json_list_care)['props']['pageProps']['workList']
            typeList = []  # 案例类型数据
            for i in list_care:
                di = {}
                if i['dic_series_category']['id'] is None:
                    di['id'] = 0
                    di['name'] = '无类型'
                else:
                    di['id'] = i['dic_series_category']['id']
                    di['name'] = i['dic_series_category']['name']
                typeList.append(di)
            run_function = lambda x, y: x if y in x else x + [y]
            item = ShiguangTypeItem()
            item['typeList'] = reduce(run_function, [[], ] + typeList)  # 案例类型数据去重
            # print(json.dumps(item['typeList'],ensure_ascii=False))
            item['companyId'] = self.companyId
            item['userId'] = self.userId
            # 发送post案例类型接口(传递给pipelines.py管道中的ShiguangTypeIdPipeline)
            yield item
            # 处理列表数据
            for i in list_care:
                yield scrapy.Request(
                    url='www.xxx.com/{}'.format(i['uuid']),  #需要修改详情页地址  {}中是详情页url中的uuid
                    callback=self.crae
                )

    def crae(self, response):
        """
            title : 组装page数据，并替换时光盒子的案例类型id
            return : Request
        """
        if response.status == 200:
            item = ShiguangCareItem()
            json_care = response.xpath('//*[@id="__NEXT_DATA__"]/text()').get()
            list_Details = json.loads(json_care)['props']['pageProps']['workDetails']
            id = list_Details['dic_series_category']['id']
            item['page'] = list_Details
            item['companyId'] = self.companyId
            item['userId'] = self.userId
            yield item # 将数据传递给pipelines.py管道中的ShiguangTypeIdPipeline

